﻿using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Net.Http;
using System.Threading;
using System.Threading.Tasks;

class AsyncWebCrawler
{
    // 配置参数
    const int MaxConcurrency = 3;      // 最大并发数
    const int MaxRetries = 2;          // 最大重试次数
    const int RequestTimeout = 5000;   // 请求超时（毫秒）
    static readonly Random random = new Random();

    // 随机User-Agent列表
    static readonly string[] UserAgents = {
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15",
        "Mozilla/5.0 (Linux; Android 10; SM-G981B) AppleWebKit/537.36"
    };

    // 模拟目标URL列表（使用httpbin测试）
    static readonly string[] Urls = {
        "https://httpbin.org/delay/1",
        "https://httpbin.org/delay/2",
        "https://httpbin.org/status/500",
        "https://httpbin.org/status/404"
    };

    static async Task Main(string[] args)
    {
        var stopwatch = Stopwatch.StartNew();
        using var httpClient = CreateHttpClient();
        var semaphore = new SemaphoreSlim(MaxConcurrency);

        var tasks = new List<Task<CrawlResult>>();
        foreach (var url in Urls)
        {
            tasks.Add(ProcessUrlAsync(httpClient, semaphore, url));
        }

        var results = await Task.WhenAll(tasks);

        int success = 0, failed = 0;
        foreach (var result in results)
        {
            if (result.Success)
            {
                Console.WriteLine($"Success: {result.Url} ({result.ElapsedMs}ms)");
                success++;
            }
            else
            {
                Console.WriteLine($"Failed: {result.Url} - {result.ErrorMessage}");
                failed++;
            }
        }

        Console.WriteLine($"\nTotal: {results.Length}, Success: {success}, Failed: {failed}");
        Console.WriteLine($"Total time: {stopwatch.Elapsed.TotalSeconds:F2}s");
    }

    static HttpClient CreateHttpClient()
    {
        var handler = new HttpClientHandler
        {
            MaxConnectionsPerServer = 20,
            AutomaticDecompression = System.Net.DecompressionMethods.All
        };

        return new HttpClient(handler)
        {
            Timeout = TimeSpan.FromMilliseconds(RequestTimeout)
        };
    }

    static async Task<CrawlResult> ProcessUrlAsync(HttpClient client, SemaphoreSlim semaphore, string url)
    {
        var result = new CrawlResult { Url = url };
        var stopwatch = Stopwatch.StartNew();

        try
        {
            await semaphore.WaitAsync();
            await Task.Delay(random.Next(0, 500)); // 随机延迟

            for (int retry = 0; retry < MaxRetries; retry++)
            {
                try
                {
                    using var request = new HttpRequestMessage(HttpMethod.Get, url);
                    request.Headers.UserAgent.ParseAdd(UserAgents[random.Next(UserAgents.Length)]);

                    using var response = await client.SendAsync(request);
                    result.StatusCode = (int)response.StatusCode;

                    if (response.IsSuccessStatusCode)
                    {
                        result.Content = await response.Content.ReadAsStringAsync();
                        result.Success = true;
                        break;
                    }
                }
                catch (Exception ex) when (IsRetryableError(ex))
                {
                    if (retry == MaxRetries - 1)
                    {
                        result.ErrorMessage = ex.Message;
                    }
                }
            }
        }
        catch (Exception ex)
        {
            result.ErrorMessage = ex.Message;
        }
        finally
        {
            semaphore.Release();
            stopwatch.Stop();
            result.ElapsedMs = stopwatch.ElapsedMilliseconds;
        }

        return result;
    }

    static bool IsRetryableError(Exception ex)
    {
        return ex is HttpRequestException ||
               ex is TaskCanceledException;
    }
}

class CrawlResult
{
    public string Url { get; set; }
    public bool Success { get; set; }
    public string Content { get; set; }
    public string ErrorMessage { get; set; }
    public long ElapsedMs { get; set; }
    public int StatusCode { get; set; }
}